{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ResNet Model Latency Testing\n",
    "\n",
    "Testing ResNet model with the default Seldon Tensor and Tensorflow Tensor.\n",
    " \n",
    "<img src=\"dog.jpeg\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'../../../proto/tensorflow/tensorflow' -> './tensorflow'\r\n",
      "'../../../proto/tensorflow/tensorflow/core' -> './tensorflow/core'\r\n",
      "'../../../proto/tensorflow/tensorflow/core/framework' -> './tensorflow/core/framework'\r\n",
      "'../../../proto/tensorflow/tensorflow/core/framework/types.proto' -> './tensorflow/core/framework/types.proto'\r\n",
      "'../../../proto/tensorflow/tensorflow/core/framework/resource_handle.proto' -> './tensorflow/core/framework/resource_handle.proto'\r\n",
      "'../../../proto/tensorflow/tensorflow/core/framework/tensor_shape.proto' -> './tensorflow/core/framework/tensor_shape.proto'\r\n",
      "'../../../proto/tensorflow/tensorflow/core/framework/tensor.proto' -> './tensorflow/core/framework/tensor.proto'\r\n"
     ]
    }
   ],
   "source": [
    "!cp ../../../proto/prediction.proto ./proto\n",
    "!cp -vr ../../../proto/tensorflow/tensorflow .\n",
    "!python -m grpc.tools.protoc -I./ --python_out=./ --grpc_python_out=./ ./proto/prediction.proto"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2018-12-04 16:46:00--  https://storage.googleapis.com/inference-eu/models_zoo/resnet_V1_50/saved_model/saved_model.pb\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 216.58.212.112, 2a00:1450:4009:807::2010\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|216.58.212.112|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 102619858 (98M) [application/octet-stream]\n",
      "Saving to: ‘model/saved_model.pb’\n",
      "\n",
      "model/saved_model.p 100%[===================>]  97.87M  26.1MB/s    in 3.9s    \n",
      "\n",
      "2018-12-04 16:46:04 (25.3 MB/s) - ‘model/saved_model.pb’ saved [102619858/102619858]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!mkdir -p model\n",
    "!wget -O  model/saved_model.pb https://storage.googleapis.com/inference-eu/models_zoo/resnet_V1_50/saved_model/saved_model.pb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wrap inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Requirement already satisfied: tensorflow in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (1.10.1)\n",
      "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (3.6.1)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (0.31.1)\n",
      "Requirement already satisfied: numpy<=1.14.5,>=1.13.3 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (1.14.5)\n",
      "Requirement already satisfied: tensorboard<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (1.10.0)\n",
      "Requirement already satisfied: setuptools<=39.1.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (39.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (1.11.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (0.7.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (1.1.0)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (0.2.0)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/site-packages (from tensorflow->-r requirements.txt (line 1)) (0.6.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/site-packages (from tensorboard<1.11.0,>=1.10.0->tensorflow->-r requirements.txt (line 1)) (3.0.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.10 in /usr/local/lib/python3.6/site-packages (from tensorboard<1.11.0,>=1.10.0->tensorflow->-r requirements.txt (line 1)) (0.14.1)\n",
      "You are using pip version 18.0, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build -E environment_grpc . seldonio/seldon-core-s2i-python36:0.13 seldon-resnet2.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4803ea44a3f8070cc7f2b44d284780d3ceccdbb27a5faeba9f884b71c0102ff3\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"resnet\" -d --rm -p 5000:5000 -v ${PWD}/model:/model seldon-resnet2.4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "import base64\n",
    "from proto import prediction_pb2\n",
    "from proto import prediction_pb2_grpc\n",
    "import grpc\n",
    "import numpy as np\n",
    "import pickle\n",
    "import tensorflow as tf\n",
    "import cv2\n",
    "import datetime\n",
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def image_2_vector(input_file):\n",
    "    nparr = np.fromfile(input_file, dtype=np.float32)\n",
    "    print(\"nparr\",nparr.dtype,nparr.shape)\n",
    "    img = cv2.imdecode(nparr, cv2.IMREAD_ANYCOLOR)\n",
    "    print(\"img\",img.dtype,img.shape)\n",
    "    print(\"Initial size\",img.shape)\n",
    "    image = cv2.resize(img, (w, h))\n",
    "    print(\"image\",image.dtype)\n",
    "    print(\"Converted size\",image.shape)\n",
    "\n",
    "    vector = image.reshape((w * h * 3))\n",
    "    print(\"vector shape\",vector.shape, \"vector type\", vector.dtype )\n",
    "    return vector\n",
    "\n",
    "def image_2_bytes(input_file):\n",
    "    with open(input_file, \"rb\") as binary_file:\n",
    "        # Read the whole file at once\n",
    "        data = binary_file.read()\n",
    "\n",
    "        #data = data.tobytes()\n",
    "        #print(data)\n",
    "        print(\"binary data size:\", len(data), type(data))\n",
    "    return data\n",
    "\n",
    "def run(function,image_path,iterations=1):\n",
    "    w = 224\n",
    "    h = 224\n",
    "\n",
    "    # NOTE(gRPC Python Team): .close() is possible on a channel and should be\n",
    "    # used in circumstances in which the with statement does not fit the needs\n",
    "    # of the code.\n",
    "    with grpc.insecure_channel('localhost:5000') as channel:\n",
    "        stub = prediction_pb2_grpc.ModelStub(channel)\n",
    "        print(\"seldon stub\", stub)\n",
    "        start_time = datetime.datetime.now()\n",
    "        processing_times = np.zeros((0),int)\n",
    "\n",
    "        img = cv2.imread(image_path)\n",
    "        print(\"img type\", type(img))\n",
    "        print(\"img\",img.shape)\n",
    "        print(\"Initial size\",img.shape)\n",
    "        image = cv2.resize(img, (w, h))\n",
    "        image = image.reshape(1, w, h, 3)\n",
    "        print(\"image\",image.dtype)\n",
    "        print(\"Converted size\",image.shape)\n",
    "        \n",
    "        if function == \"tensor\":\n",
    "            datadef = prediction_pb2.DefaultData(\n",
    "                names = 'x',\n",
    "                tensor = prediction_pb2.Tensor(\n",
    "                    shape = image.shape,\n",
    "                    values = image.ravel().tolist()\n",
    "                )\n",
    "            )\n",
    "        elif function == \"tftensor\":\n",
    "            print(\"Create tftensor\")\n",
    "            datadef = prediction_pb2.DefaultData(\n",
    "                names = 'x',\n",
    "                tftensor = tf.make_tensor_proto(image)\n",
    "            )\n",
    "            \n",
    "        GRPC_request = prediction_pb2.SeldonMessage(\n",
    "            data = datadef\n",
    "        )\n",
    "            \n",
    "        for I in range(iterations):\n",
    "            start_time = datetime.datetime.now()\n",
    "            response = stub.Predict(request=GRPC_request)\n",
    "            end_time = datetime.datetime.now()\n",
    "            duration = (end_time - start_time).total_seconds() * 1000\n",
    "            processing_times = np.append(processing_times,np.array([int(duration)]))\n",
    "            \n",
    "        print('processing time for all iterations')\n",
    "        for x in processing_times:\n",
    "            print(x,\"ms\")\n",
    "        print('processing_statistics')\n",
    "        print('average time:',round(np.average(processing_times),1), 'ms; average speed:', round(1000/np.average(processing_times),1),'fps')\n",
    "        print('median time:',round(np.median(processing_times),1), 'ms; median speed:',round(1000/np.median(processing_times),1),'fps')\n",
    "        print('max time:',round(np.max(processing_times),1), 'ms; max speed:',round(1000/np.max(processing_times),1),'fps')\n",
    "        print('min time:',round(np.min(processing_times),1),'ms; min speed:',round(1000/np.min(processing_times),1),'fps')\n",
    "        print('time percentile 90:',round(np.percentile(processing_times,90),1),'ms; speed percentile 90:',round(1000/np.percentile(processing_times,90),1),'fps')\n",
    "        print('time percentile 50:',round(np.percentile(processing_times,50),1),'ms; speed percentile 50:',round(1000/np.percentile(processing_times,50),1),'fps')\n",
    "        print('time standard deviation:',round(np.std(processing_times)))\n",
    "        print('time variance:',round(np.var(processing_times)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldon stub <proto.prediction_pb2_grpc.ModelStub object at 0x7fdfd646db70>\n",
      "img type <class 'numpy.ndarray'>\n",
      "img (224, 224, 3)\n",
      "Initial size (224, 224, 3)\n",
      "image uint8\n",
      "Converted size (1, 224, 224, 3)\n",
      "processing time for all iterations\n",
      "86 ms\n",
      "78 ms\n",
      "89 ms\n",
      "84 ms\n",
      "78 ms\n",
      "77 ms\n",
      "78 ms\n",
      "77 ms\n",
      "74 ms\n",
      "81 ms\n",
      "79 ms\n",
      "75 ms\n",
      "80 ms\n",
      "79 ms\n",
      "77 ms\n",
      "75 ms\n",
      "75 ms\n",
      "76 ms\n",
      "75 ms\n",
      "76 ms\n",
      "83 ms\n",
      "77 ms\n",
      "80 ms\n",
      "79 ms\n",
      "78 ms\n",
      "77 ms\n",
      "77 ms\n",
      "76 ms\n",
      "74 ms\n",
      "77 ms\n",
      "74 ms\n",
      "79 ms\n",
      "78 ms\n",
      "75 ms\n",
      "75 ms\n",
      "78 ms\n",
      "79 ms\n",
      "76 ms\n",
      "80 ms\n",
      "75 ms\n",
      "78 ms\n",
      "76 ms\n",
      "76 ms\n",
      "81 ms\n",
      "78 ms\n",
      "78 ms\n",
      "75 ms\n",
      "77 ms\n",
      "77 ms\n",
      "74 ms\n",
      "75 ms\n",
      "81 ms\n",
      "75 ms\n",
      "75 ms\n",
      "76 ms\n",
      "78 ms\n",
      "72 ms\n",
      "79 ms\n",
      "81 ms\n",
      "80 ms\n",
      "74 ms\n",
      "82 ms\n",
      "77 ms\n",
      "77 ms\n",
      "77 ms\n",
      "78 ms\n",
      "75 ms\n",
      "77 ms\n",
      "77 ms\n",
      "77 ms\n",
      "75 ms\n",
      "79 ms\n",
      "76 ms\n",
      "80 ms\n",
      "78 ms\n",
      "75 ms\n",
      "76 ms\n",
      "76 ms\n",
      "79 ms\n",
      "77 ms\n",
      "76 ms\n",
      "76 ms\n",
      "80 ms\n",
      "77 ms\n",
      "74 ms\n",
      "79 ms\n",
      "75 ms\n",
      "73 ms\n",
      "77 ms\n",
      "76 ms\n",
      "78 ms\n",
      "78 ms\n",
      "76 ms\n",
      "77 ms\n",
      "76 ms\n",
      "76 ms\n",
      "75 ms\n",
      "74 ms\n",
      "77 ms\n",
      "82 ms\n",
      "processing_statistics\n",
      "average time: 77.3 ms; average speed: 12.9 fps\n",
      "median time: 77.0 ms; median speed: 13.0 fps\n",
      "max time: 89 ms; max speed: 11.2 fps\n",
      "min time: 72 ms; min speed: 13.9 fps\n",
      "time percentile 90: 80.1 ms; speed percentile 90: 12.5 fps\n",
      "time percentile 50: 77.0 ms; speed percentile 50: 13.0 fps\n",
      "time standard deviation: 3.0\n",
      "time variance: 7.0\n"
     ]
    }
   ],
   "source": [
    "run(\"tensor\",\"./dog.jpeg\",iterations=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldon stub <proto.prediction_pb2_grpc.ModelStub object at 0x7fdfd646dac8>\n",
      "img type <class 'numpy.ndarray'>\n",
      "img (224, 224, 3)\n",
      "Initial size (224, 224, 3)\n",
      "image uint8\n",
      "Converted size (1, 224, 224, 3)\n",
      "Create tftensor\n",
      "processing time for all iterations\n",
      "75 ms\n",
      "71 ms\n",
      "78 ms\n",
      "71 ms\n",
      "73 ms\n",
      "81 ms\n",
      "81 ms\n",
      "76 ms\n",
      "73 ms\n",
      "78 ms\n",
      "74 ms\n",
      "71 ms\n",
      "70 ms\n",
      "73 ms\n",
      "79 ms\n",
      "72 ms\n",
      "77 ms\n",
      "68 ms\n",
      "77 ms\n",
      "76 ms\n",
      "71 ms\n",
      "73 ms\n",
      "71 ms\n",
      "69 ms\n",
      "72 ms\n",
      "74 ms\n",
      "70 ms\n",
      "73 ms\n",
      "71 ms\n",
      "68 ms\n",
      "72 ms\n",
      "71 ms\n",
      "69 ms\n",
      "70 ms\n",
      "69 ms\n",
      "72 ms\n",
      "70 ms\n",
      "77 ms\n",
      "66 ms\n",
      "67 ms\n",
      "77 ms\n",
      "71 ms\n",
      "72 ms\n",
      "70 ms\n",
      "72 ms\n",
      "68 ms\n",
      "71 ms\n",
      "69 ms\n",
      "71 ms\n",
      "70 ms\n",
      "71 ms\n",
      "73 ms\n",
      "70 ms\n",
      "70 ms\n",
      "75 ms\n",
      "70 ms\n",
      "72 ms\n",
      "70 ms\n",
      "74 ms\n",
      "72 ms\n",
      "72 ms\n",
      "74 ms\n",
      "71 ms\n",
      "73 ms\n",
      "73 ms\n",
      "72 ms\n",
      "75 ms\n",
      "72 ms\n",
      "69 ms\n",
      "70 ms\n",
      "71 ms\n",
      "70 ms\n",
      "71 ms\n",
      "68 ms\n",
      "69 ms\n",
      "70 ms\n",
      "73 ms\n",
      "70 ms\n",
      "69 ms\n",
      "75 ms\n",
      "69 ms\n",
      "74 ms\n",
      "71 ms\n",
      "72 ms\n",
      "69 ms\n",
      "69 ms\n",
      "73 ms\n",
      "71 ms\n",
      "70 ms\n",
      "66 ms\n",
      "75 ms\n",
      "71 ms\n",
      "69 ms\n",
      "70 ms\n",
      "70 ms\n",
      "73 ms\n",
      "72 ms\n",
      "71 ms\n",
      "70 ms\n",
      "69 ms\n",
      "processing_statistics\n",
      "average time: 71.8 ms; average speed: 13.9 fps\n",
      "median time: 71.0 ms; median speed: 14.1 fps\n",
      "max time: 81 ms; max speed: 12.3 fps\n",
      "min time: 66 ms; min speed: 15.2 fps\n",
      "time percentile 90: 76.0 ms; speed percentile 90: 13.2 fps\n",
      "time percentile 50: 71.0 ms; speed percentile 50: 14.1 fps\n",
      "time standard deviation: 3.0\n",
      "time variance: 8.0\n"
     ]
    }
   ],
   "source": [
    "run(\"tftensor\",\"./dog.jpeg\",iterations=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The stats illustrate that the tftensor payload which is the only difference improves on the latency performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm -f resnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
